# -*- coding: utf-8 -*-
'''
Miscellaneous classes and functions.
These are not very specific in usage, however could be useful anywhere.
'''
import os, sys, time, math, gzip, json, shutil
import traceback, paramiko, fnmatch, requests
import numpy as np
from StringIO import StringIO
from gpsdatetime import GPSDateTime, datetime
from ftplib import FTP
import MySQLdb as sql
from lxml import etree

PI = math.pi
rad2deg = 180./PI
deg2rad = PI/180.

_path_ = os.path.dirname(os.path.abspath(__file__))

scinda_vars = {
    'west'    : (2, '-', (0, 1.2), 'S4', 'UHF West Antenna'),
    'west_r'  : (3, '-', (0, 1.2), 'S4', 'UHF West Antenna Redundant'),
    'L'       : (5, '-', (0, 1.2), 'S4', 'L-Band Antenna'),
    'east'    : (6, '-', (0, 1.2), 'S4', 'UHF East Antenna'),
    'east_r'  : (7, '-', (0, 1.2), 'S4', 'UHF East Antenna Redundant'),
    'drift'   : (14, '.r', (-100, 300), 'Velocity m/s',
                 'Drift for West Channels')}

class LoginError(Exception):
    pass

class my_float(float):
    pass

class my_str(str):
    pass

class LISNRequest(object):
    '''
    Class to access django api information, uses requests module
    '''
    
    def __init__(self):
        self.params = Config()
        self.client = requests.session()
        self.login()
    
    def login(self):
        self.client.get(self.params.url_login)
        csrftoken = self.client.cookies['csrftoken']
        login_data = {'username': self.params.username, 'password': self.params.password, 'next': '/',
                      'this_is_the_login_form': '1', 'csrfmiddlewaretoken': csrftoken}
        req = self.client.post(self.params.url_login, data=login_data)
        #TODO check for successful login
        if '<a href="/accounts/logout">[logout]' in req.text:
            self.ok = True
        else: 
            self.ok = False
            
    def stations(self, **kwargs):
        url = os.path.join(self.params.url, 'utils/stations')        
        req = self.client.get(url, params=kwargs)
        return req.json()
    
    def station(self, code, instrument):
        url = os.path.join(self.params.url, 'utils/station')
        params = {'code':code, 'instrument':instrument}
        req = self.client.get(url, params=params)
        return req.json()
    
    def update_station(self, code, instrument, **kwargs):
        url = os.path.join(self.params.url, 'utils/update', instrument, code)
        req = self.client.get(url, params=kwargs)
        return req.json()
        

class MySQL(object):
    '''
    '''
    def __new__(cls, host, user, passwd, db, verbose=False):

        mysql = object.__new__(cls)
        mysql.conn = sql.connect(host   = host,
                                    user   = user,
                                    passwd = passwd,
                                    db     = db)
        mysql.cursor = mysql.conn.cursor()
        mysql.verbose = verbose
        return mysql

    def query(self, query):
        '''
        '''
        result = self.cursor.execute(query)
        if self.verbose:
            print '>>>%s -> %s' % (query,result)
        self.commit()
        return result

    def select(self, table, fields, filter='1', distinct=False):
        '''
        '''
        ret    = []
        fields = ','.join([s for s in fields])
        if distinct:
            query  = 'SELECT DISTINCT %s FROM %s WHERE %s' % (fields, table, filter)
        else:
            query  = 'SELECT %s FROM %s WHERE %s' % (fields, table, filter)
        

        if self.verbose:
            print '>>>%s -> ' % query,

        self.cursor.execute(query)
        result = self.cursor.fetchall()
        if self.verbose:
            print result
        for tup in result:
            if '*' in fields:
                ret.append(list(tup))
            else:
                ret.append(dict(zip(fields.split(','), list(tup))))
        return ret

    def update(self, table, fields, values, filter='1'):
        '''
        '''
        vars = ""
        for field, value in zip(fields, values):
            if type(value) in (float,):
                vars += "%s=%f," % (field, value)
            elif type(value) in (int, long):
                vars += "%s=%d," % (field, value)
            elif value in ('NULL', 'null', 'Null', 'False'):
                vars += "%s=NULL," % field
            else:
                vars += "%s='%s'," % (field, value)
        vars = vars[:-1]
        query = '''UPDATE %s SET %s WHERE %s''' % (table, vars, filter)
        
        if self.verbose:
            print '>>>%s -> ' % query,
        result = self.cursor.execute(query)
        if self.verbose:
            print result
        self.commit()
        return result

    def insert(self, table, fields, values):
        '''
        '''
        
        fields = ','.join(fields)
        vars   = ""
        for var in values:
            if type(var) in (float,):
                vars += "%f," % var
            elif type(var) in (int, long):
                vars += "%d," % var
            elif var in ('NULL', 'null', 'Null', False, 'False'):
                vars += "NULL,"
            else:
                vars += "'%s'," % var
        vars = vars[:-1]
        query  = '''INSERT INTO %s (%s) VALUES (%s)''' % (table, fields, vars)
        result = self.cursor.execute(query)
        if self.verbose:
            print '>>>%s -> %s' % (query,result)
        self.commit()
        return result

    def select_one(self, table, field, fields, values, like=False):
        '''
        '''
        
        filter = ""
        for fild, value in zip(fields, values):
            if type(value) in (float,):
                filter += "%s=%f AND " % (fild, value)
            elif type(value) in (int, long):
                filter += "%s=%d AND " % (fild, value)
            elif value in ('NULL', 'null', 'Null'):
                filter += "%s=NULL AND " % fild
            elif value.startswith('@str:'):
                filter += "%s = '%s' AND " % (fild, value[5:])
            elif value.startswith('@var:'):
                filter += "%s=%s AND " % (fild, value[5:])
            elif like:
                filter += "%s LIKE '%%%s%%' AND " % (fild, value)
            else:
                filter += "%s = '%s' AND " % (fild, value)
        filter = filter[:-4]

        query = '''SELECT %s FROM %s WHERE %s''' % (field, table, filter)

        self.cursor.execute(query)
        result = self.cursor.fetchall()

        if self.verbose:
            print '>>>%s -> %s' % (query,result)

        if len(result)==1:
            return result[0][0]
        elif len(result)>1:
            return [tup[0] for tup in result]
        else:
            return False

    def commit(self):
        self.conn.commit()

    def close(self):
        self.conn.commit()
        self.cursor.close()
        self.conn.close()

class MyXML(object):
    '''
    '''

    def __init__(self, s=''):
        '''
        '''

        if s:
            self.data  = etree.fromstring(s, )            
        else:
            self.data  = etree.Element('data')

    def __str__(self):
        '''
        '''
        
        return self.tostring()

    def update_attr(self, attr, value):
        '''
        '''
        self.data.set(attr, value)

    def delete_attr(self, attr):
        '''
        '''
        #print dir(self.doc)
        if attr in self.data.keys():
            del self.data.attrib[attr]

    def find_text(self, element, tag, text):
        '''
        '''
        for el in element:
            if el.tag.strip()==tag.strip() and el.text.strip()==text.strip():
                return el
        return False

    def add_date(self, year, month, day, dt=None):
        '''
        Append or add new childs for given year, month and day to XML data
        '''    
        if dt:
            year,month,day = dt.year,dt.month,dt.day
        # add year
        el_year = self.find_text(self.data, 'y', '%s' % year)
        if el_year is False :
            el_year = etree.SubElement(self.data, 'y')
            el_year.text = '%s' % year
        # add month
        el_month = self.find_text(el_year, 'm', '%s' % month)
        if el_month is False:
            el_month = etree.SubElement(el_year, 'm')
            el_month.text = '%s' % month
        # add day
        el_day = self.find_text(el_month, 'd', '%s' % day)
        if el_day is False:
            el_day = etree.SubElement(el_month, 'd')
            el_day.text = '%s' % day
            return True
        else:
            return False

    def delete_data(self):
        '''
        '''
        etree.strip_elements(self.data, 'y', 'm', 'd')

    def tostring(self, pretty_print=True, encoding='utf-8'):
        '''
        '''
        s = etree.tostring(self.data, pretty_print=pretty_print,
                           encoding=encoding, xml_declaration=True)
        return s.replace("'",'"')

class Stations(dict):
    '''
    Class to hold lisn stations info
    '''
    vars = ['idstation', 'station', 'country', 'latitude', 'longitude', 'altitude', 'idinstrument',
            'idstation', 'old_code', 'lisn_code', 'code', 'instrument', 'network', 'server', 'status',
            'last_file', 'last_file_date', 'receiver', 'antenna', 'receiver_number', 'antenna_number',
            'receiver_firmware', 'reported']

    def __init__(self, host='lisn.igp.gob.pe', user='lisn', passwd='lisn',
                 local_file='./stations.info', verbose=False):
        self.verbose = verbose
        self.data = {}
        self.lisn_codes = {}
        self.local = False
        self.local_file = local_file
        print self.local_file
        if os.path.exists(self.local_file):
            self.local = True
            fo = open_file(self.local_file)
            for line in fo:
                code = line.split(',')[10]
                self.lisn_codes[line.split(',')[9]] = code
                self.data[code] = [s.strip() for s in line.split(',')]
            fo.close()
        
        if host == 'local':
            print 'Using local data from ./stations.info'
            self.ok = False
        else:
            self.sql = MySQL(host, user, passwd, 'lisn', verbose=verbose)
            self.ok = True

    def __getattr__(self, name):
        if not name in self:
            raise AttributeError()
        return self[name]

    def read(self, label, instrument, position=(0, 0, 0)):
        '''
        Update dictionary with new values for the given code
        '''
        self.clear()
        result = False
        if self.ok:
            filter = '''
(code='%s' or lisn_code='%s')
AND instrument='%s'
AND stations.idstation=instruments.idstation
''' % (label, label, instrument)
            result = self.sql.select('stations,instruments', ['*'], filter=filter)
        if result:
            result = list(result[0])
            for key in self.vars:
                self[key] = result[self.vars.index(key)]

            if self['lisn_code']:
                self['lcode'] = self['lisn_code']
            else:
                self['lcode'] = self['code']
            self['fullname'] = '%s, %s' % (self['station'].upper(), self['country'])
            self['location'] = (self['latitude'], self['longitude'],
                                self['altitude'])
            if self['network'].upper() in ('LISN', 'UNIVAP'):
                self.data[self['code']] = [str(x) for x in result]
        else:            
            if label in self.data or label in self.lisn_codes:
                if label in self.lisn_codes:
                    label = self.lisn_codes[label]
                result = self.data[label]
                for key in self.vars:
                    self[key] = result[self.vars.index(key)]
                self['latitude'] = float(self['latitude'])
                self['longitude'] = float(self['longitude'])
                self['altitude'] = float(self['altitude'])
                self['lcode'] = self['lisn_code']
                self['fullname'] = '%s, %s' % (self['station'].upper(), self['country'])
                self['location'] = (self['latitude'], self['longitude'],
                                    self['altitude'])                
            else:
                self['code']     = label
                self['lcode']    = label
                self['station']  = label
                self['fullname'] = label
                self['location'] = position
                self['latitude'] = position[0]
                self['longitude']= position[1]
                self['receiver'] = 'Unknown'

    def get(self, field, server=None, network=None, country=None, code=None, instrument=None):
        '''
        Return a list of values for the given field
        '''
        if self.ok:
            query = ''' s.idstation=i.idstation '''
            if code:
                query += ''' AND code='%s' ''' % code
            if server:
                query += ''' AND server LIKE '%s' ''' % server
            if network:
                query += ''' AND network LIKE '%s' ''' % network
            if country:
                query += ''' AND country LIKE '%s' ''' % country
            if instrument:
                query += ''' AND instrument LIKE '%s' ''' % instrument

            ret = self.sql.select(table='stations s,instruments i', fields=[field], filter=query, distinct=True)
            return [s[field] for s in ret]
        else:
            ret = []

            for key in self.data:
                data = dict(zip(self.vars, self.data[key]))
                flag = False
                if network and server and network.lower().strip()==data['network'].lower().strip() and server.lower().strip()==data['server'].lower().strip():
                    flag = True
                elif network and server is None and network.lower().strip()==data['network'].lower().strip():
                    flag = True
                elif server and network is None and server.lower().strip()==data['server'].lower().strip():
                    flag = True

                if flag:
                    ret.append(data[field])

            return ret

    def codes(self, networks=[], servers=[], instrument=None):
        '''
        Return a list of the codes for the given networks
        '''

        raise DeprecationWarning('Use get method')
    
    def read_all(self, instrument='gps'):
        codes = self.get('code', network='lisn', instrument=instrument) + self.get('code', network='univap', instrument=instrument)
        for code in codes:
            self.read(code, instrument=instrument)

    def updatedb(self, code, dates, instrument, datatype):
        if self.ok:
            if instrument.lower() not in ('gps', 'ionosonde', 'magnetometer'):
                query = '''datatypes.iddatatype=products.iddatatype
AND product LIKE '%s'
AND alias LIKE '%s' ''' % (instrument, datatype)
                data = self.sql.select('products,datatypes', ['idproduct', 'dates'], query)[0]
                xml = MyXML(data['dates'])
                if isinstance(dates, datetime):
                    dates = [dates]
                for dt in dates:
                    xml.add_date(dt.year, dt.month, dt.day)

                self.sql.update('products', ['dates'], [xml.tostring(False)], "idproduct=%s" % data['idproduct'])
            else:
                query = '''datatypes.iddatatype=data.iddatatype
AND data.idinstrument=instruments.idinstrument
AND code = '%s'
AND instrument LIKE '%s'
AND alias LIKE '%s' ''' % (code, instrument, datatype)
                data = self.sql.select('data,datatypes,instruments', ['iddata', 'dates'], query)[0]
                xml = MyXML(data['dates'])
                if isinstance(dates, datetime):
                    dates = [dates]
                for dt in dates:
                    xml.add_date(dt.year, dt.month, dt.day)

                self.sql.update('data', ['dates'], [xml.tostring(False)], "iddata=%s" % data['iddata'])

    def close(self):
        self.read_all()
        if self.ok:
            self.sql.close()
        if self.ok and self.local:
            fo = open(self.local_file, 'wb')
            for line in self.data.values():            
                fo.write(','.join(line))
                fo.write('\n')
            fo.close()

class MyFTP(object):
    '''
    Class to manage sftp and ftp connections with servers, functions supported:
    listdir, download, getfileutime
    '''

    def __init__(self, host, username, password, secure=False, havekey=False):
        if secure:
            self.secure = secure
        else:
            self.secure = False
        self.transport = 0
        self.ftp = 0
        self.error = False
        try:
            print 'Connecting to ' + host
            if secure:
                self.transport = paramiko.Transport((host, 22))
                if havekey:
                    sha = False
                    try:
                        print 'Trying SHA key...'
                        privatekeyfile = os.path.expanduser('~/.ssh/id_rsa')
                        mykey = paramiko.RSAKey.from_private_key_file(privatekeyfile)
                        self.transport.connect(username=username, pkey=mykey)
                        sha = True
                    except:
                        pass
                    if not sha:
                        try:
                            print 'Trying DSA key...'
                            privatekeyfile = os.path.expanduser('~/.ssh/id_dsa')
                            mykey = paramiko.DSSKey.from_private_key_file(privatekeyfile)
                            self.transport.connect(username=username, pkey=mykey)
                        except:
                            print 'Error connecting with key'
                            self.error = True
                else:
                    self.transport.connect(username=username, password=password)
                self.ftp = paramiko.SFTPClient.from_transport(self.transport)
            else:
                self.ftp = FTP(host)
                self.ftp.login(username, password)
        except:
            traceback.print_exc(2)
            self.error = True

    def listdir(self, path='.'):
        if self.secure:
            return self.ftp.listdir(path)
        else:
            return [x.split('/')[-1] for x in self.ftp.nlst(path)]

    def download(self, remotepath, localpath):
        print 'Downloading: %s' % remotepath
        if self.secure:
            self.ftp.get(remotepath, localpath)
        else:
            self.ftp.retrbinary('RETR '+remotepath, open(localpath, 'wb').write)

    def upload(self, localpath, remotepath):
        print 'Uploading: %s' % localpath
        if self.secure:
            self.ftp.put(localpath, remotepath)
        else:
            self.ftp.storbinary('STOR '+remotepath, open(localpath, 'rb'))

    def remove(self, path):
        print 'Deleting: %s' % path
        if self.secure:
            self.ftp.remove(path)
        else:
            self.ftp.delete(path)
    
    def exists(self, filename):
        path = os.path.dirname(filename)
        name = filename.split('/')[-1]
        return name in self.listdir(path)

    
    def utime(self, path):
        if self.secure:
            return self.ftp.stat(path).st_mtime
        else:
            return int(time.mktime(time.strptime(str(self.ftp.sendcmd('MDTM '+ path)[-14:]),
                                                 '%Y%m%d%H%M%S')))
    
    def open(self, filename, mode='rb'):        
        if self.exists(filename):
            if self.secure:
                return self.ftp.open(filename, mode)
            else:
                so = StringIO()
                self.ftp.retrbinary('RETR '+filename, so.write)
                return so
        else:
            print 'File not found in server: %s' % filename
            return None
    
    def close(self):
        if self.ftp <> 0: self.ftp.close()
        if self.transport <> 0: self.transport.close()

class scindaData(list):
    '''
    class to ...
    '''

    def __init__(self, filename, vars='all'):
        '''
        '''
        if vars=='all':
            vars = ['west', 'west_r', 'east', 'east_r', 'L', 'drift']
        elif vars=='west':
            vars = ['west', 'west_r']
        elif vars=='east':
            vars = ['east', 'east_r']
        else:
            vars = [var for var in vars.split(',') if var in scinda_vars]
        self.vars = [scinda_vars[var] for var in vars]

        fo = open_file(filename)
        while True:
            try:
                line = fo.next()
            except StopIteration:
                break
            if line[0] == '#' or len(line)==0: continue
            cols = [x.strip() for x in line.split(' ') if len(x)<>0]
            dt = GPSDateTime(cols[0]+cols[1])
            cols = [float(x.strip()) for x in line.split(' ') if len(x)<>0]
            self.append((dt,)+tuple(cols[2:]))
        fo.close()

class s4Data(list):
    '''
    '''

    def __init__(self, filename, vars='all'):
        '''
        '''
        self.prns = set()
        self.arcs = {}
        fo = open_file(filename)
        self.header = fo.next()
        while True:
            try:
                line = fo.next()
            except StopIteration:
                break
            if line[0] == '#' or len(line)==0: continue
            rec = s4Record(line)
            self.append(rec)
            prns = self[-1].keys()
            self.checkbreak(prns, rec.epoch)
            self.prns = self.prns.union(prns)
        self.endphase(self.prns)
        self.check()
        fo.close()
        self.date = self[-1].epoch

    def checkbreak(self, prns, epoch):
        '''
        '''
        for prn in prns:
            if prn not in self.arcs:
                self.arcs[prn] = [[len(self)-1, len(self)-1]]
            else:
                if epoch-self[self.arcs[prn][-1][-1]].epoch>60:
                    self.endphase(prn)
                    self.arcs[prn] += [[len(self)-1, len(self)-1]]
                else:
                    self.arcs[prn][-1][-1] = len(self)-1

    def endphase(self, prn):
        '''
        '''
        if isinstance(prn, (list, tuple, set, dict)):
            [self.endphase(p) for p in prn]
        elif prn in self.arcs and self.arcs[prn][-1][1] is None:
            self.arcs[prn][-1][1] = len(self)-1

    def check(self):
        '''
        '''
        for prn in self.arcs:
            self.arcs[prn] = [arc for arc in self.arcs[prn] \
                                   if arc[1]-arc[0]>=10]

    def iterlist(self, prn, keys=['s4']):
        '''
        '''
        def chooser(rec, prn, keys):
            tup = (rec.epoch,)
            for key in keys:
                tup += (rec[prn][key],)
            return tup

        rec_idx = []
        for arc in self.arcs[prn]:
            rec_idx += [x for x in range(arc[0], arc[1])]
        
        for x in rec_idx:
            yield chooser(self[x], prn, keys)
        return

class s4Record(dict):
    '''
    '''
    def __init__(self, line):
        '''
        '''
        cols = [x.strip() for x in line.split(' ') if len(x)<>0]
        tot_sec = int(cols[2])
        fyear   = int(cols[0])
        if fyear<100:
            fyear = fyear+2000
        else:
            fyear = fyear+1900
        fdoy    = int(cols[1])
        fhour   = tot_sec/3600
        fmin    = (tot_sec-(fhour*3600))/60
        fsec    = tot_sec-(fhour*3600)-(fmin*60)
        self.epoch = GPSDateTime(year=fyear, doy=fdoy, hour=fhour, minute=fmin,
                                 second=fsec)

        for nsat in range(int(cols[3])):
            if len(cols)<nsat*4+7:
                break
            prn = int(cols[nsat*4+4])
            if prn not in self:
                self[prn] = {}
            self[prn]['s4'] = float(cols[nsat*4+5].replace(',','.'))
            self[prn]['az'] = float(cols[nsat*4+6].replace(',','.'))
            self[prn]['el'] = float(cols[nsat*4+7].replace(',','.'))

    def __getitem__(self, index):
        '''
        Allow you to access GPS satellites, eg record['G13'], as
        simply record[13].  For GLONASS or Galileo, you must use the full code.
        '''
        if index == 'epoch':
            return self.epoch
        else:
            return dict.__getitem__(self, index)

class magData(list):
    '''
    '''
    vars =  ('DD', 'MM', 'YYYY', 'hh', 'mm', 'D(deg)', 'H(nT)', 
             'Z(nT)', 'I(deg)', 'F(nT)', 'T1(deg)', 'T2(ºC)')

    def __init__(self, url, station={}):
        '''
        '''
        self.arcs = []
        self.station = station
        self.header = ''
        #self.code = url.split('/')[-1][:4]
                
        try:              
            print 'Opening: %s' % url
            fo = open_file(url)
            self.header = fo.next()
        except BaseException as e:
            print e
            return
        
        while True:
            try:                
                values = fo.values(fmt=(int,int,int,int,int,float,float,float,float,float))                
            except StopIteration:
                break
            except (ValueError, TypeError):
                continue

            if not values or len(values)<>10: continue
            self.append(magRecord(values))
            self.checkbreak()
        fo.close()
        if len(self)>0:
            self.date = self[-1].epoch.date()
        else:
            self.date = None
    
    def checkbreak(self):
        '''
        '''

        if not self.arcs:
            self.arcs.append([len(self)-1, len(self)-1])
        elif self[-1].epoch-self[self.arcs[-1][-1]].epoch>60:
            self.arcs.append([len(self)-1, len(self)-1])
        else:
            self.arcs[-1][-1] = len(self)-1
        
    def iterlist(self, keys=['H']):
        '''
        '''
        if len(keys)==1:
            return [rec[keys[0]] for rec in self]
        else:
            return [[rec[key] for key in keys]for rec in self]
        
    def save(self, filename, date=[]):
        '''
        '''
        if isinstance(date, (list, tuple)):
            date = [dt.date() for dt in date]
        elif isinstance(date, (datetime, GPSDateTime)):
            date = [GPSDateTime(date).date()]
        else:
            date = []            
        
        vars = self[0].keys()
        if len(vars)==5:
            fmt = '%s%9.4f%9.1f%9.1f%9.4f%9.1f\n'
            values = ('D', 'H', 'Z', 'I', 'F')
        elif len(vars)==6:
            fmt = '%s%9.4f%9.1f%9.1f%9.4f%9.1f%9.2f\n'
            values = ('D', 'H', 'Z', 'I', 'F', 'T1')
        elif len(vars)==7:
            fmt = '%s%9.4f%9.1f%9.1f%9.4f%9.1f%9.2f%9.2f\n'
            values = ('D', 'H', 'Z', 'I', 'F', 'T1', 'T2')
        fo = open(filename, 'wb')
        fo.write('%s\n\n' % self.header)
        fo.write('%3s%3s%5s%3s%3s%9s%9s%9s%9s%9s%9s%9s\n\n' % self.vars)
        cnt = 0
        for rec in self:
            if date and rec.epoch.date() not in date:
                continue
            allvalues = (rec.epoch.strftime(' %d %m %Y %H %M'),)+tuple([rec[x] for x in values])
            fo.write(fmt % allvalues)
            cnt += 1
        fo.close()
        
        print '%s of %s records saved at %s' % (cnt, len(self), filename)
        
    def merge(self, data_i):
        '''
        Merge current magdata object with another.
        '''

        if isinstance(data_i, str):
            data_i = magData(data_i)
        data_j = self[:]
        self.arcs = []
        
        del self[:]
        
        i = 0
        j = 0
        imax = len(data_i)
        jmax = len(data_j)

        while True:
            if i==imax or j==jmax:
                break          
            if data_i[i]==data_j[j]:
                self.append(data_j[j])
                i+=1
                j+=1
            elif data_i[i]>data_j[j]:
                self.append(data_j[j])
                j+=1
            else:
                self.append(data_i[i])
                i+=1 
            self.checkbreak()            
            
        if i==imax:
            temp = data_j[j:]
        else:
            temp = data_i[i:]
        
        for rec in temp:
            self.append(rec)
            self.checkbreak()
        
        self.header = data_i.header            

        print 'Merging %d + %d -> %d records' % (jmax, imax, len(self))
    
class magRecord(dict):
    '''
    '''
    def __init__(self, values):
        '''
        '''  
        self.epoch = GPSDateTime(values[2],values[1],values[0],values[3],values[4])
        self['D'] = values[5]
        self['H'] = values[6]
        self['Z'] = values[7]
        self['I'] = values[8]
        self['F'] = values[9]
        try:
            self['T1'] = values[10]
            self['T2'] = values[11]
        except IndexError:
            pass

    def __getitem__(self, index):
        '''
        Allow you to access GPS satellites, eg record['G13'], as
        simply record[13].  For GLONASS or Galileo, you must use the full code.
        '''
        if index == 'epoch':
            return self.epoch
        else:
            return dict.__getitem__(self, index)
        
    def __eq__(self, other):
        '''
        Equal comparison reference to the epoch of the record
        '''        
        return self.epoch==other.epoch

    def __ne__(self, other):
        '''
        Not equal comparison reference to the epoch of the record
        '''
        return self.epoch!=other.epoch

    def __lt__(self, other):
        '''
        Comparison reference to the epoch of the record
        '''
        return self.epoch<other.epoch

    def __le__(self, other):
        '''
        Comparison reference to the epoch of the record
        '''
        return self.epoch<=other.epoch

    def __gt__(self, other):
        '''
        Comparison reference to the epoch of the record
        '''
        return self.epoch>other.epoch

    def __ge__(self, other):
        '''
        Comparison reference to the epoch of the record
        '''
        return self.epoch>=other.epoch
    
def fitlin(X, Y):
    '''
    '''
    n = float(len(X))
    xbar = sum(X)/n
    ybar = sum(Y)/n
    sumxx = sum([i**2 for i in X])
    sumxy = sum([X[i]*Y[i] for i in range(len(X))])
    b = (sumxy - n*xbar*ybar)/(sumxx-n*xbar**2)
    a = ybar - b*xbar
    return a, b

def fix_jumps(Y, X, vmax, j=None, id=None):
    '''
    Fix jumps of a list Y
    '''

    if j:
        start = j
        end = j+1
    else:
        start = 1
        end = len(Y)

    for i in range(start, end):
        diff = abs(Y[i] - Y[i-1])
        if diff>vmax:
            if i>=start+4:
                i1 = i-10
                if i1<1: i1 = 1
                i2 = i-1
                x1 = [X[x]-X[i1] for x in range(i1, i2)]
                y1 = [Y[x] for x in range(i1,i2)]
                a, b = fitlin(x1, y1)
                ynew  = a + b*(X[i]-X[i1])
                ydiff = ynew - Y[i]
            else:
                psum = sum([Y[x] for x in range(i)])
                ynew  = psum/i
                ydiff = ynew - Y[i]
            #if id:
                #print 'Jump found: id=%s, x=%.2f, y1=%.3f, y0=%.3f, diff =%.3f, ydiff =%.3f'% \
                #    (id, X[i], Y[i], Y[i-1], diff, ydiff)
            for k in range(i,len(Y)):
                Y[k] += ydiff
    return Y

def fmod(x):
    fmodx = x
    while fmodx < 0 or fmodx > 2*PI:
        if fmodx >= 0:
            fmodx = fmodx - 2*PI
        else:
            fmodx = fmodx + 2*PI
    return fmodx

def sign(a, b):
    if b>=0:
        return abs(a)
    else:
        return -abs(a)

def mnbrak(ax, bx, F, args):
    '''
    Find a range where a zero of the function F(x, *args) is located
    ax, bx are just guest (initial values)
    '''

    golden = 1.618
    glimit = 100.
    tiny = 1e-18
    dum = 0

    fa = F(ax, *args)
    fb = F(bx, *args)

    if fb>fa:
        dum, ax, bx = ax, bx, dum
        dum, fb, fa = fb, fa, dum

    cx = bx+golden*(bx-ax)
    fc = F(cx, *args)

    while fb>fc:
        r = (bx-ax)*(fb-fc)
        q = (bx-cx)*(fb-fa)
        u = bx-((bx-cx)*q-(bx-ax)*r)/(2.0*sign(max(abs(q-r), tiny), q-r))
        ulim = bx+glimit*(cx-bx)
        if (bx-u)*(u-cx) > 0.0:
            fu = F(u, *args)
            if fu < fc:
                ax, bx = bx, u
                fa, fb = fb, fu
                break
            elif fu > fb:
                cx=u
                fc=fu
                break
            u=cx+golden*(cx-bx)
            fu = F(u, *args)
        elif (cx-u)*(u-ulim) > 0.0:
            fu = F(u, *args)
            if fu < fc:
                bx, cx = cx,u
                u=cx+golden*(cx-bx)
                fb, fc = fc, fu
                fu=F(u, *args)
        elif (u-ulim)*(ulim-cx) >= 0.0:
            u=ulim
            fu = F(u, *args)
        else:
            u=cx+golden*(cx-bx)
            fu = F(u, *args)
        ax, bx, cx = bx, cx, u
        fa, fb, fc = fb, fc, fu
    return (ax, bx, cx), (fa, fb, fc)

def brent(val, fval, F, args, itmax=20, tol=1e-2):
    '''
    Find a zero of a function F(x, *args) using brent method.
    val = (ax, bx, cx)
    fval = (F(ax), F(bx), F(cx))

    val: is a range where is located the Zero.
    '''

    cgold = 0.381966
    zeps = 1e-10
    e = 0

    ax, bx, cx = val
    fa, fb, fc = fval

    a=min(ax, cx)
    b=max(ax, cx)

    x, w, v = bx, bx, bx
    fx, fw, fv = fb, fb, fb

    for i in range(itmax):
        xm = .5*(a+b)
        tol1 = tol*abs(x)+zeps
        tol2 = 2*tol1
        if abs(x-xm)<=(tol2-0.5*(b-a)):
            return x, fx, i
        if abs(e) > tol1:
            r=(x-w)*(fx-fv)
            q=(x-v)*(fx-fw)
            p=(x-v)*q-(x-w)*r
            q=2.0*(q-r)
            if q > 0.0: p = -p
            q=abs(q)
            etemp=e
            e=d
            if (abs(p) >= abs(0.5*q*etemp)) or (p<=q*(a-x)) or (p >=q*(b-x)):
                if x>=xm: e = a-x
                else: e = b-x
                d=cgold*e
            else:
                d=p/q
                u=x+d
                if (u-a < tol2) or (b-u < tol2):
                    d=sign(tol1, xm-x)
        else:
            if x>=xm: e = a-x
            else: e = b-x
            d=cgold*e
        if abs(d)>=tol1: u = x+d
        else: u = x+sign(tol1, d)
        fu = F(u, *args)

        if (fu <= fx):
            if (u >= x): a=x
            else: b=x
            v, w, x = w, x, u
            fv, fw, fx = fw, fx, fu
        else:
            if (u < x): a=u
            else: b=u
            if (fu <= fw) or (w == x):
                v, w = w, u
                fv, fw = fw, fu
            elif (fu <= fv) or (v == x) or (v == w):
                v=u
                fv=fu
    print 'Max iteration reach'
    return x, fx, i

def copy_file(filei, fileo, move=False):
    '''
    '''
    output_path = os.path.dirname(fileo)
    if not os.path.exists(output_path):
        os.makedirs(output_path)
    shutil.copy(filei, fileo)
    if move:
        os.remove(filei)

def rglob(src, pattern):
    '''
    Recursive glob
    '''
    ret = []
    for root, dirs, files in os.walk(src):
        ret.extend([os.path.join(root, file) for file in files \
                   if fnmatch.fnmatch(file, pattern)])
    return ret

class open_file(object):
    '''
    Wrap "sufficiently file-like objects" (ie those with readline())
    in an iterable which counts line numbers, strips newlines, and raises
    StopIteration at EOF.
    '''
    def __new__(cls, file):
        '''Create a open_file object.

        Input can be filename string, file descriptor number, or any object
        with `readline'.
        '''

        if isinstance(file, open_file):
            file.reset()
            return file
        of = object.__new__(cls)
        if isinstance(file, (str, unicode)):
            if file.endswith('.gz'):
                of.fid = gzip.open(file)
            else:
                of.fid = open(file, 'rb')
        elif isinstance(file, int):
            of.fid = os.fdopen(file)
        elif hasattr(file, 'readline'):
            of.fid = file
        else:
            raise IOError('Type ' + `type(file)` + ' is not supported.')

        if hasattr(of.fid, 'name'):
            of.name = of.fid.name
        elif hasattr(of.fid, 'filename'):
            of.name = of.fid.filename
        else:
            of.name = str(file)
        try:
            of.fid.read()
        except:
            raise IOError('Error reading file: %s' % of.name)
        of.reset()
        return of

    def next(self):
        '''
        Return the next line, also incrementing `lineno'.
        '''
        self.last_pos = self.fid.tell()
        self.line = self.fid.readline()
        if not self.line:
            raise StopIteration()
        self.lineno += 1
        return self.line.rstrip('\r\n')

    def readline(self):
        '''
        A synonym for next() which doesn't strip newlines or raise
        StopIteration.
        '''
        self.last_pos = self.fid.tell()
        self.line = self.fid.readline()
        if self.line:
            self.lineno += 1
        return self.line

    def backline(self):
        '''
        Set the file's current position to the previous line.
        '''
        self.fid.seek(self.last_pos)

    def values(self, char = ' ', fmt=None):
        '''
        Return the line as python list.
        the line is splited by char and formatted as fmt
        fmt could be int, float or a list of types 
        '''
        
        L = [s.strip() for s in self.next().split(char) if s]

        if isinstance(fmt, type):
            return [fmt(s) for s in L]
        elif isinstance(fmt, (list, tuple)):            
            return [f(s) for f,s in zip(fmt, L)]
        else:
            return L
                    

    def read(self, n=None):
        self.lineno = None
        if n is None:
            return self.fid.read()
        else:
            return self.fid.read(n)

    def write(self, s):
        return self.fid.write(s)

    def find(self, s):
        return self.fid.find(s)

    def seek(self, pos, start=0):
        self.lineno = None
        self.fid.seek(pos)#, start)

    def tell(self):
        return self.fid.tell()

    def __iter__(self):
        return self

    def reset(self):
        '''Go back to the beginning if possible. Set lineno to 0 regardless.'''
        if hasattr(self.fid, 'seek'):
            try:
                self.fid.seek(0)
            except IOError:
                pass
        self.lineno = 0

    def close(self):
        '''Close the file.  A closed file cannot be used for further I/O.'''
        if hasattr(self.fid, 'fileno') and self.fid.fileno() < 3:
            # Closing stdin, stdout, stderr can be bad
            return
        if hasattr(self.fid, 'close'):
            try:
                self.fid.close()
            except IOError:
                pass

class time_file(list):
    '''
    Class to save and update files time into a file
    '''

    def __init__(self, filename):
        self.file = filename
        self.status = {}
        if os.path.exists(filename):
            for x in list(open(filename)):
                self.append(x)

    def find(self, name):
        for time_info in self:
            if name in time_info:
                return time_info.split(',')[1]
            else: continue
        return '0'

    def update(self, name, file_time):
        index = [self.index(x) for x in self if name in x]
        if len(index)<>0:
            self[index[0]] = '%s,%i,\n' % (name, file_time)
        else:
            self.append('%s,%i,\n' % (name, file_time))

    def save(self):
        fo = open(self.file, 'w')
        fo.write(''.join(self))
        fo.close()

    def update_status(self, site, utime):
        if site not in self.status:
            self.status[site] = int(utime)

    def save_status(self, file):
        fo = open(file, 'w')
        for site, utime in self.status.items():
            fo.write('%s\n%s\n' % (site,
                                   GPSDateTime.fromtimestamp(utime).strftime('%Y-%m-%d %H:%M:%S')))
        fo.close()

class Stats(object):
    '''
    Class to calculate statistical values from a list
    '''
    def __init__(self, L):
        self.n = float(len(L))
        self.sum = sum(L)
        if len(L)>1:
            self.average = self.sum/self.n
            self.variance = sum([(x-self.average)**2 for x in L])/(self.n-1)
            self.std_dev = self.variance**0.5
        elif len(L)==1:
            self.average = L[0]
            self.variance = 0
            self.std_dev = 0
        else:
            raise RuntimeError('List is empty')

class _set(list):
    '''
    A simplified set() implementation for old python versions
    '''
    def __init__(self, iterable=None):
        self._data = {}
        if iterable:
            self._update(iterable)
    def __len__(self):
        return len(self._data)
    def __repr__(self):
        return 'set(%r)' % self._data.keys()
    def __iter__(self):
        return self._data.iterkeys()
    def __contains__(self, item):
        return item in self._data
    def _update(self, iterable):
        try:
            iterable = iter(iterable)
            for item in iterable:
                self._data[item] = True
        except:
            raise('%r is not iterable' % iterable)

    def add(self, item):
        self._data[item] = True

    def union(self, other):
        result = self.__class__(self)
        result._update(other)
        return result

    def update(self, iterable):
        self._update(iterable)

class Config(object):
    '''
    Read configuration file
    '''
    def __init__(self, filename='/etc/lisn.cfg'):
        if not os.path.exists(filename):
            raise LISNError('Configuration file does not exists')
        self.all = json.load(open(filename))
       
    def username(self):
        return self.all['user']['username']
    username = property(username)
    
    def password(self):
        return self.all['user']['password']
    password = property(password)
    
    def url(self):
        return self.all['server']['url']
    url = property(url)
    
    def url_login(self):
        return self.all['server']['url_login']
    url_login = property(url_login)
    
    def mag_server(self, server=None):
        if server is not None:
            return self.all['mag_server'][server]
        else:
            return self.all['mag_server']
    
    def gps_server(self, server=None):
        if server is not None:
            return self.all['gps_server'][server]
        else:
            return self.all['gps_server']
    
if sys.version_info[1]<4:
    set = _set
else:
    set = set

if __name__=='__main__':
    con = Config()
    print con.mag_server()
    
    